{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "import mxnet as mx\n",
    "from mxnet import init, gluon, nd, autograd, image\n",
    "from mxnet.gluon import nn\n",
    "import numpy as np\n",
    "import pickle as p\n",
    "import matplotlib.pyplot as plt\n",
    "from time import time\n",
    "%matplotlib inline\n",
    "ctx = mx.gpu()\n",
    "data_dir = '/home/sinyer/python/data/cifar10'"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "def load_cifar(route = data_dir+'/cifar-10-batches-py'):\n",
    "    def load_batch(filename):\n",
    "        with open(filename, 'rb')as f:\n",
    "            data_dict = p.load(f, encoding='latin1')\n",
    "            X = data_dict['data']\n",
    "            Y = data_dict['labels']\n",
    "            X = X.reshape(10000, 3, 32,32).astype(\"float\")\n",
    "            Y = np.array(Y)\n",
    "            return X, Y\n",
    "    def load_labels(filename):\n",
    "        with open(filename, 'rb') as f:\n",
    "            label_names = p.load(f, encoding='latin1')\n",
    "            names = label_names['label_names']\n",
    "            return names\n",
    "    label_names = load_labels(route + \"/batches.meta\")\n",
    "    x1, y1 = load_batch(route + \"/data_batch_1\")\n",
    "    x2, y2 = load_batch(route + \"/data_batch_2\")\n",
    "    x3, y3 = load_batch(route + \"/data_batch_3\")\n",
    "    x4, y4 = load_batch(route + \"/data_batch_4\")\n",
    "    x5, y5 = load_batch(route + \"/data_batch_5\")\n",
    "    test_pic, test_label = load_batch(route + \"/test_batch\")\n",
    "    train_pic = np.concatenate((x1, x2, x3, x4, x5))\n",
    "    train_label = np.concatenate((y1, y2, y3, y4, y5))\n",
    "    return train_pic, train_label, test_pic, test_label\n",
    "\n",
    "def accuracy(output, label):\n",
    "    return nd.mean(output.argmax(axis=1)==label).asscalar()\n",
    "\n",
    "def evaluate_accuracy(test_data, net, ctx):\n",
    "    acc = 0.\n",
    "    for data, label in test_data:\n",
    "        data = data.as_in_context(ctx)\n",
    "        label = label.as_in_context(ctx)\n",
    "        output = net(data)\n",
    "        acc = acc + accuracy(output, label)\n",
    "    return acc / len(test_data)\n",
    "\n",
    "def augment(data, auglist):\n",
    "    data = nd.pad(data, pad_width=(0,0,0,0,2,2,2,2),mode='constant',constant_value=0)\n",
    "    data = nd.transpose(data, (0,2,3,1))\n",
    "    temp = []\n",
    "    for d in data:\n",
    "        for aug in auglist:\n",
    "            d = aug(d)\n",
    "        temp.append(d)\n",
    "    data = nd.stack(*temp)\n",
    "    data = nd.transpose(data, (0,3,1,2))\n",
    "    return data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "train_pic, train_label, test_pic, test_label = load_cifar()\n",
    "\n",
    "batch_size = 128\n",
    "train_pic = np.transpose(train_pic.astype('float32')/255, (0,2,3,1))\n",
    "test_pic = np.transpose(test_pic.astype('float32')/255, (0,2,3,1))\n",
    "mean = [0.4914, 0.4822, 0.4465]\n",
    "std = [0.2470, 0.2435, 0.2616]\n",
    "for i in range(3):\n",
    "    train_pic[:,:,:,i] = (train_pic[:,:,:,i] - mean[i])/std[i]\n",
    "    test_pic[:,:,:,i] = (test_pic[:,:,:,i] - mean[i])/std[i]\n",
    "train_pic = np.transpose(train_pic, (0,3,1,2))\n",
    "test_pic = np.transpose(test_pic, (0,3,1,2))\n",
    "train_data = gluon.data.DataLoader(gluon.data.ArrayDataset(train_pic, train_label.astype('float32')), \n",
    "                                   batch_size, shuffle=True)\n",
    "test_data = gluon.data.DataLoader(gluon.data.ArrayDataset(test_pic, test_label.astype('float32')), \n",
    "                                  batch_size, shuffle=False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "aug_train = image.CreateAugmenter(data_shape=(3, 32, 32), rand_crop=True, rand_mirror=True)\n",
    "\n",
    "net = gluon.nn.Sequential()\n",
    "with net.name_scope():\n",
    "    net.add(\n",
    "        nn.Conv2D(channels=20, kernel_size=5, activation='relu'),\n",
    "        nn.MaxPool2D(pool_size=2, strides=2),\n",
    "        nn.Conv2D(channels=50, kernel_size=3, activation='relu'),\n",
    "        nn.MaxPool2D(pool_size=2, strides=2),\n",
    "        nn.Flatten(),\n",
    "        nn.Dense(128, activation=\"relu\"),\n",
    "        nn.Dense(10)\n",
    "    )\n",
    "net.initialize(ctx=ctx)\n",
    "loss = gluon.loss.SoftmaxCrossEntropyLoss()\n",
    "trainer = gluon.Trainer(net.collect_params(), 'nag', {'learning_rate': 0.1, 'momentum': 0.9, 'wd': 5e-4})"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0 loss:2.0119 tracc:0.2570 teacc:0.3713 time:14.268\n",
      "10 loss:1.3100 tracc:0.5544 teacc:0.5581 time:7.804\n",
      "20 loss:1.2569 tracc:0.5781 teacc:0.6130 time:7.651\n",
      "30 loss:1.1894 tracc:0.6005 teacc:0.5855 time:7.535\n",
      "40 loss:1.1639 tracc:0.6079 teacc:0.6289 time:7.671\n",
      "50 loss:0.9303 tracc:0.6849 teacc:0.7162 time:7.679\n",
      "60 loss:0.6075 tracc:0.7905 teacc:0.7745 time:7.693\n",
      "70 loss:0.5282 tracc:0.8144 teacc:0.7878 time:7.538\n",
      "80 loss:0.4438 tracc:0.8454 teacc:0.8059 time:7.599\n",
      "90 loss:0.4015 tracc:0.8602 teacc:0.8078 time:7.486\n",
      "100 loss:0.3907 tracc:0.8643 teacc:0.8119 time:7.539\n",
      "tracc:0.868638 teacc:0.811412\n"
     ]
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAXcAAAD8CAYAAACMwORRAAAABHNCSVQICAgIfAhkiAAAAAlwSFlz\nAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDIuMS4wLCBo\ndHRwOi8vbWF0cGxvdGxpYi5vcmcvpW3flQAAIABJREFUeJzt3Xl8VNX9//HXmcm+B5IQSNj3TUED\nUnDFDdBKrdriWrtItXVpa1v129a29ttvv1pba3+1fovWurXiTtGi1KIWi6AEN3bZScKShOx7Zub8\n/jiDxhBggECYm/fz8ZhHMnfu3PncXHjPmXPPPWOstYiIiLf4uroAERHpfAp3EREPUriLiHiQwl1E\nxIMU7iIiHqRwFxHxoIOGuzHmEWNMqTFm1X4eN8aY3xtjNhpjPjLGnNT5ZYqIyKGIpOX+KDDtAI9P\nB4aGb7OBB4+8LBERORIHDXdr7WKg4gCrzAQet84yIMMY07uzChQRkUMX0wnbyAOK2twvDi/b2X5F\nY8xsXOue5OTkk0eMGNEJLy8i0n2sWLGi3FqbfbD1OiPcTQfLOpzTwFo7B5gDUFBQYAsLCzvh5UVE\nug9jzLZI1uuM0TLFQN829/OBHZ2wXREROUydEe7zgWvCo2YmAdXW2n26ZERE5Ng5aLeMMeYp4Ewg\nyxhTDPwUiAWw1v4fsACYAWwEGoCvHq1iRUQkMgcNd2vt5Qd53ALf7rSKRETkiOkKVRERD1K4i4h4\nkMJdRMSDFO4iIh6kcBcR8SCFu4iIByncRUQ8SOEuIuJBCncREQ9SuIuIeJDCXUTEgxTuIiIepHAX\nEfEghbuIiAcp3EVEPEjhLiLiQQp3EREPUriLiHiQwl1ExIMU7iIiHqRwFxHxIIW7iIgHxXR1ASIi\nxytrLat31LBmRw31LQEaWoIkx/nJy0yid3oCIWupbQpQ09hKWV0zZbXNVDe24vcZYv0+WgIhKupb\nqGxoob45QEswREsgxM1nD+XCE/oc1doV7iJy3LLW0hwI0dASpKElQHMgRFNrkGDI4vcZYnw+giFL\nY2uQptYg1Y2tVDa0UFnfwu6aZnbXNFFR34INby9kLaGQJWgtfmOIj/UTH+OjqTVITWOAxtYg/Xsm\nMaxXKvExPl5ZtYst5fUR1+szkBIfQ8hCazBErN9Hj+Q4MpPjSIn3k+6PJS7GR2pC7NH5g7WhcBeR\nTtMUDtmQhWDIEh/rIzkuBr/PYK2loSVIVWMr28rr2bKnnp1VTTS0BGlsDVBe10JJZSMlVY00tgQJ\nWRfC1h78dTuSnhhLbloCPZLj8PsMAMaAzxj8PkMwZGkOBKltCpAY62dAVhJxMX62ltfz5LJttARD\nfG5QT755+iCmDMkiJT6GxDg/dc0BSiob2VndSIzPR2pCDKkJsWSnxn/mtbqawl1EDll1YysvvlfM\n+t21bNhdR1FlA1UNrTQHQh2unxDruihC7YLaZyApzoVmZlIs+ZlJnNQ/g5T4WHwG/D5DYpyfpFg/\nSXExxMf6SIj14zeGoLUEgq4FnxDrIzHWT1piLJlJcWQkxZIQ6z/s/dv7aSAlft+ITIj1k5USz4l9\nMw57+8eCwl1EIhYKWZ4pLOKeheupqG+hR3IcQ7JTOH1oNpnJcaQnxpIY68fvM/gMNAdC1DW7vur4\nGB/J8TGkJcQyoGcSA7KSyU1LwHectHTb8vtMh8EeTaK7ehE5ZqobWrn20Xd5f3sVEwZk8vjXJjIm\nL72ry5L9ULiLyEG1BkPc8NcVrCqp5rdfOpGLx+dhzPHX4pZPKdxF5ICstfxk3ire3rSH31x2Il88\nKb+rS5IIKNxFpEPBkGXdrhrmvV/C3OVFfPuswVxysoI9WijcRbqp0tomVu+oYd3OWrZX1LO9ooHS\nmmZC4bGHu2uaqWsOADBzXB9uPXd4V5Yrh0jhLuIB1lo+3l1HjN+QnuiGAVbWt1Be18z2igbW7qxl\n7c4aSmubqWtupaYxQHVj6yfPz0qJo2+PJAZnp7hx2gYmD47j5P6ZnNw/k/zMRPWxRxmFu0gUq21q\n5YX3Snh86VY2le3/SspYv2FITip5GQmkxKeQkhDDwKwURvdJY1SfNNKOwRWTcmwp3EWiRFNrkPW7\nallZUs2qkmpWllSzflctgZDlxPx0/ufisSTH+6lubKWxJUhmchxZKXHkpiUyJCeFuJijPE9goBkq\ntkBrAyT1gKSeYHxueaDZLW+pg2ArpOVBSi+wQShZAVsWu3X6jIe8kyCt3bwrFVvcerW7oG6321ZC\nOsSnQWwS+Pzgj3Xb7DEIUnJgx/uw5S3YvfrTx2OTICEDEjMhLtkt88e6mlobIdgMSVmQmuvqB7Ah\naKiAik1QsRmCLe5141LAF45Qnw/S+7rXTkiHPZtgzwb3s2obVG5zy3PHQK+xMOQcyBpyVA9HROFu\njJkG3A/4gYettf/b7vF+wGNARnid2621Czq5VhHPs9ZSUtXIymIX3h/vrqOsrpnyWjdPSiB8iWd6\nYiwn5Kdz3emDOH90LuOO1dWSVduh/GOo2Qk1JVBVBNXbXXhVF7kgjFRMAhg/tNYDxr0R2KB7LLEH\nZA11bwI73ofKLZ8+zx8PcUnQVPPp+vtj/JA1zG072OLeFBqrwq95GBIzISYRmmvdGxUHmRshqSdk\nDoA+46CxEta/Au8/CZ+/v+vD3RjjBx4AzgWKgeXGmPnW2jVtVvsx8Iy19kFjzChgATDgKNQr4gkt\ngRBLNpWzpayelqCbDGvdzlre215JaW0zADE+w6DsZHqlJTA4K5k+GYmMyUtjdJ/0Y98Hbi0s/QO8\ndudnAzwlFzL6Qf4EOPFy6DnYtWgbK6Bhj3teTALExLnlcckucGuKoXIrBFpgwBQYcJpbb9dKF+Zl\n66B8AxQXQq9RcMr1MOBUyOjrWs3GuG231EOgCUIBF941O90bQc0O6DUG+k2ChLR99yfQHG6pt7rn\n+eMgNtG14uvL3SeExgr3OsbnXrPHIPeJpO3fZO/EN8EW98ZXucW9efQcDD2HQGK7N11r3bZjEzv9\nELUXSct9IrDRWrsZwBgzF5gJtA13C+z9C6YDOzqzSJFoVt3Yyjub91DZ0EJ1Yyvrdtby2trd1DYF\nPrNevx5JTBmSxUn9MjghP4PhualHND9Kp2lpgPk3warnYORFMOlbkNYbUntDTHznvla/U9wtEsZA\nfIq77ZXRL7Lnx8Tvv/b0PHeL5PX3vsH6EiB7mLsd7DlpvQ++7U4QSbjnAUVt7hcD7f96PwP+aYy5\nCUgGzuloQ8aY2cBsgH79+h1qrSJRIRiybK9oYGVJNf/4aAdvrCujJfhpazc9MZZpo3OZMbY34/pm\nEB/rI87vI8Z/hH3ioaDrW47Ehn/BpkVQ9C7sXuX6jmMSILM/zHwAcka69apL4Kkvw65VMPUncNqt\nnwaaHNciCfeOjmT7jqbLgUettb8xxnwOeMIYM8baz3bAWWvnAHMACgoKDnMiT5HjT11zgJc/3MHz\n7xXzUXH1J7MjZqfGc9Wk/swYm0vvjETSE2NJjvN3TpdKoBl2fgQbX3N9ubtXu+6Lc37mukE6Yi28\n+Sv4992u77jPeCj4mut6aG2EdS/Dw+fAJQ+7k4p/m+W6Pq54Boadd+Q1yzETSbgXA33b3M9n326X\nrwPTAKy1S40xCUAWUNoZRYocj1qDIZZsLGf+hzt4ddUuGlqCDMlJ4epJ/Rmem8rw3FRG90k/8vm9\nQ0HX97zzQzdSpG43lK6FXR+5vl7jg/yJMGomLHsAit6B6XfD5jfdybvmWjjlmzDhG/Dve+CdB2Hc\nVXDhffu+CZx2K8y9Ap663HVbJOfA1190/d4SVYw9yEz4xpgY4GPgbKAEWA5cYa1d3WadV4CnrbWP\nGmNGAouAPHuAjRcUFNjCwsJO2AWRY8day/tFVbz4Xgn/WLmTivoWUhNimDGmN1+e2JfxfTPcR931\nr8B7j8Fp34e+Ez7dQGsTbPuP6+6o3QnVxe5WU+JCONDsTg6m9HKjLGIT3HC+pir3fOOD5GzoMRjy\nT4a8AncyMjk8bG/1i/D3m6Cl1t0fcJo7ebfhn67rJRRwfebn/dIN3+tIayP841ZX0xcfcsMK5bhh\njFlhrS046HoHC/fwxmYAv8MNc3zEWvtLY8xdQKG1dn54hMxDQAquy+aH1tp/HmibCneJJo0tQZ5e\nvp1H397K1j0NxMf4OGdUL2ae2IczhmcTb0JuDPTOj1zrecf7gHEhff1bLiBbGuCJi6FoWXirxi1P\nz3dD/hLSXb+3z+9Ge1Rtg6Zq6D/FjYvuP9kF+8H61Ss2w8ZFMORsN8IDXJ/5sgfdCb/JN6vfPIp1\nargfDQp3iQa1Ta08vnQbj/xnC3vqWzipXwazJuQzoz+kVK1zF99sDV8oEwqPfsnoB2fc5obiPTIN\n8gvgyufg2Wvh41fh87+DwVPdMML99Y2L7Eek4a4rVEU6UNcc4NElW3jorS1UN7Zy5vBsfjiyglEf\n3AYLN7ix1eDGR+dPhMk3QfYId8FM7lg3Xhpcv/a86+GPk9wY6Bn3wsnXdtl+SfehcBdpY0t5PU8s\n3cazK4qobQpwzsgcbjl7GGMb34Wnv+LGdk/4BvQYCFnDXav8QBekjLvcneBc8RfXmp943bHbGenW\nFO4iwJ66ZubOfZTMra8wxVfNNUmN9OiXR9qgyVC8Av75Yzf2++oXITnr0DY+414YfxXknXx0ihfp\ngMJdur0FH+1g44v/zY2hv9Ecn0pMj77EpuZC9TZY9C+3Ur/PwRVPu5Oeh8of41r4IseQwl26rYaW\nAD99/j1OWfMLbvYvpmbIRaTNmvPZbpaGCjfHSe8T3bBEkSihcJduacPuWu55/EW+U3svo/3bCJ1+\nG2ln3bHvEMGkHpHPdSJyHFG4S7ezbGMpix//OQ/45kJSOlw8F9/w6V1dlkinUrhL92EtJcvn03PB\nj/ihr4imwdNIuPgPkJLd1ZWJdDqFu3QPzbU0P/UV8rYuoohcyqc/RNbEy3SlpnjWUf7eLZHjQKAZ\nO/dKYra+wd2hq6n52n/IOuVLCnbxNIW7eFsoCC/Mxmz5Nz9omc3Ai25jdD91w4j3KdzF2167E9bM\n47X8m3jZdyYzxh6bb8ER6WoKd/Eua+GDv2JHfYGf75nKqUOySInXaSbpHhTu4l315dBYye70Eymu\nbOS8Ub26uiKRY0bhLt5Vtg6AJdVZ+Ayco3CXbkThLt5Vvh6AeSUpFPTvQVbKfr7tXsSDFO7iXWXr\nCcUm89buOM4brVa7dC8Kd/GusvWUJw4ADOeNyu3qakSOKYW7eFfZeta09mZEbir9eiZ1dTUix5TC\nXbypsQrqdvFefQ6nDOzR1dWIHHMKd/Gm8o8BWB3ozZCclC4uRuTYU7iLN5W5kTIbbR6DFe7SDSnc\nxZvK1hH0xVFkcxiSrXCX7kfXYos3la2nLK4fycE4slM1vl26H7XcxZvK17PZ5DMoJwWjqX2lG1K4\ni/e01EPVdla19FKXjHRbCnfxnvINALzfmMvgnOQuLkakayjcxXvajpRRy126KYW7eE/5ekLGz1ab\nq3CXbkvhLseX7cvgoamu3/xw7VpJRUI/rC+W/pp2QLophbsXVW2HpprDf35zLcw5C7a93Xk17fXB\nU/DeE/t//N2HoGQF7F59eNsPhaDoHdbGjKR/zyRi/fonLt2T/uV7SXEhPHUF/G4s/ON7h7+drf+B\nHe/BqhcOfxsVm2HN313Y7lW7C166BebfCB/8bd/ntDbBx6+638PTBxyy8vXQVM2y1iHqkpFuTRcx\necW/74E3fgkJGdBzKGx6w32H6OGM8d7ylvu5femhP3fNfHh3DmwNb+Pz98PJ17rfl/weQgHInwjz\nb4LUXBg89dPnbloELXXu9/CIl0O2fRkAC2sGcO44hbt0X2q5H2+a61wL91CEQrD8zzDoTPjuaphy\nCzSUfzJq5JBtWex+7l7tZleM1M4P4ZmroboIpv4Y+n0OXrsT6kqhrgwKH4ETvgRXPQdZw+Hpa2D3\namqaWllZXE1o9TxIzIQeg2DPxkMqua45wAdFVVD0DsHEnmwM9VLLXbq1iMLdGDPNGLPeGLPRGHP7\nftb5kjFmjTFmtTGmg8/cEpGF/wX/dyq0NET+nJIVULcLxl0J8Skw4FS3fG/rub2WetizqePHGipg\n90oYeAZgoeidyOvY8E/38+v/gtN/AJ//PbQ2Enr1Dure/B020MSOsd+icFeQv4/+HbU2nu1zvswp\nP3+ZS/7wBs2rXyY0/ELIGXXAlntpTRPVDa0AhEKW51YUc9a9b/KFB5awZ+1iyjLGAUazQUq3dtBu\nGWOMH3gAOBcoBpYbY+Zba9e0WWcocAcwxVpbaYzJOVoFe96Wf0N9GXw0Fwq+9uny7cug1xgX3u2t\newl8MTD0PHc/cwCk5bu+84nXfXbdonfhheuguhiuXwI5Iz77+Nb/uJ+nfhe2LXFdM8POd8s+nOu6\nXGY9BakdfG3dxteh94mQkk19c4Bl5emEelzBuaseJdbG8FJoEjc/XAQUATA19joe8f+KJwcsZEva\nySSub+DB8jF8s98OfB8vhGAA/OF/oovuom5LIa80j+UPxYPZZnPonZ5IYpyfzWX1jOubwaXD4+i5\nqoR7S84EYFC2LmCS7iuSPveJwEZr7WYAY8xcYCawps061wEPWGsrAay1pZ1daLdQsxMqtwIGlv4R\nTroWfD7Y/CY8PhPGXgaXPPzZ51gLa1+GAadBYoZbBDBgCmbjIrCWjWV1PLd8G/1X/j++3PQMdQm5\nJPoSKH32B8wbdR8xPsPQXikMzUklf8tiTGyya/33Hgfbwv3uoRC8+Suo3ErNo5fxzOgHafUlMCI3\nlWG5qdRX72FI0Tsszb2KXz+whJUl1QRDlh7x53JC/CJ6tRQRd9YP+U36MDKTYxmSnUpe5gx4pZyT\nlz/EyaFVNMWk8ZuNvUlsLufaUCtvr1hBVWI/Vhbt4eblf8QXCnGZ+TeXxUPQxFITzGBXUw7bp9/D\nuadNxrf+ZVgF7zOcPukJpCXEHrNDJ3K8iSTc89jb1HKKgVParTMMwBizBPADP7PWvtp+Q8aY2cBs\ngH79+h1Ovd5W5E4GMukGWPZH2PgaDDwdXvoOYGDlc7RMuZUPGntR0D8Tn8+4fvWKTdSf9E2eX7qV\nJRvLWba5ghmtGfwqppxr7nmCxZU9uTrmX1weM5dXfGfyg6qruNK/iDsCT7Hkted5OzTmkxJei3+F\nqphh/OnJD7nejuCkkqf5uKiUDe++yucrt/Js4HQuKX+L3De+y02tN2HDPXvn+5bzp7ggDxT1JybP\ncP0Zg5g0qCenDOxJXPVIKF3DtJFT99llzr0LNr8Bu1eSMO4qvp85hpcXbubaOHh43j95PXQSJ/q3\ncltsI6+M+G+mnH4eaSWL8VcXk1lfTubalxi56Zdw2j/cpxt/PP/7rWtotBorIN1bJP8DOhpuYTvY\nzlDgTCAfeMsYM8Za+5mzcdbaOcAcgIKCgvbbiH61u2Hjv+DEWeDzY63d/4yEgWZ3EvSkqyE+1S3b\n/g7EJMLUn7hhhEv/4Pq8K7cQ/OIjhP7+bf495/tc1/AtbjhzMLdNGwHrXgbgq0uzeHfPavIyEjl/\ndC/6+8+DD//E9OSNTD3lZK5e/jJkTWH6tfOYWN/Crj2nE3juLZ5Meonaq29hQ3kD27ZvZejrxTyf\nPo2te+r5055ePBTbyp1/fJzrYv5BZWwmVWf/mu21L3Dhe3dzzmmnsXLYt1m/q5ZT171IsCSFR++4\nkbj4dlPs9hzsbh2JS4KL58CTF8O4K7h+wGCuHXcV3PczfnFqHN85YQoji4pgIUyf/gXI6At5wz99\nft+J8NLN8MFf3d+qz3j65mQe4YEUiX6RhHsx0LfN/XxgRwfrLLPWtgJbjDHrcWG/vFOqjAa718Df\nvgTVRbTWlvLd4jMo3FrJXTNHc97o3H3Xf+f/3EiSQBOcFh6Tvn0p5Be4wJs4G/71U+zWJWzo/Xmu\nW9iDLzWfyw0xL/HVIdfy4JubOCEvnWnrXmZL/EhWVCby2NcmcPrQLPeGYi1syePynG3gfxXqS+HL\nT4Ix9EyJp2dKLzjv5/D810n/+FkKTrqagtrXAbjkkiu4JO9k6ipHwv2/4VdD1zGo6APMlO9y3Vkj\nwN4Bwe0kLP0tE0ZfwIRTxsPSZTD4TPztgz0S+SfDbds+GbaZkJ4NST3JCxSTl58By96F1D6Qnr/v\nc8dfDR8+Ba/9xI00+ty3Dv31RTwoktEyy4GhxpiBxpg4YBYwv90684CzAIwxWbhums2dWehxbeMi\neOR8CLYS6Hcq9vVfsm7lcuJifMx+YgU3/u091u2qYU9dM8GQdSNS3vqNe+77T7KzqoElq7dgd30E\n/SZhraWw50U0m0SqQkl8ecsFZKXEM/KL/4WJS+HHqfMZ1zeD3zy7CLPjfZ6pO5EfzRjJGcOyP/2k\nYIzrN9+yGJbcD8OmQ792vWljLoH8CW7M+dwr3YVF8WmQeyIAKZk5kD2SwdufdR/f9o5XNwam3w0p\nveDvN7ohk9VFMOTsw/8btv+E03MolIeHQ25/x9Xe0acgnw8u/J0L9lAr9J10+DWIeMhBW+7W2oAx\n5kZgIa4//RFr7WpjzF1AobV2fvix84wxa4Ag8ANr7Z6jWfhxIdgKi++Fxb/GZg9n3dQ/8/NXNvFH\n+yHP9nqclG+/wXMLFtJrxa18sCaNOwNfpdXE8qvkp/lSoJalOVcypfSv3Hz3g8SZVqbEhbi9MJll\nK95k654Gpsb/gJOH9edvU89jZO8095pVN+BffA8vxLyGz9cIQGj4BXx1yoB96xtwKnz0NGDg7J/s\n+7gxcMUz7lPEu3OgsRKGTft0hApAv0lQthaGng8Zbc6TJKTDhffBU7Pg2WvdssFHEO7tZQ2Bjxe6\nUT01xdD3pv2vmzPCffpZcr+rV0Qiu0LVWrsAWNBu2Z1tfrfA98K36FW51bUSd33k5lc5+05Izvr0\n8S2LXb9uxgB3sc3rv4CdH7Cpz4Xc1ngNhY9uISU+jp2n/Q+j/3MT/PlsLt/1EaGENHwtNZyZXctr\nfb/DJSsXsDB2KreXnMvSuBf4Rf/3ScgeSGilj6qe48i3Sdw0dSjTx55PUly7QzT5Jmipx2cMO5vj\neasyk+9dfkHHfft7x7uf8CXoNbrjfU7qAWf9F0y+GVa/6Pqw2xp4Gqz4C0z4xr7PHT7dtf5XPQ9Z\nwyCzf6R/6YPLGgbvP+kCHvb91NHemXe4rqykHp1Xg0gUMy6Xj72CggJbWFjYJa/doR0fwMPnuI/2\nMQkQCrqgu3oexMS54YhPXuIunw+r8aVzW/PXeCU4gZG907hiYl9mjs9zQ/BemO0uxf/ct9wVoxte\ng3k3uO36Y+Gm97BpfTAv3QIrn3UX7gSa4Yb/dO5+rVsA/T/n3owORygExcv3H651ZfDgZBh/FZzz\n08Ovs711C2Du5ZB3MpSuhduLPvuJQqSbMsassNYWHGw9/W8Bd/Lx1TtcV8M18yB7pGvFvvANeOUH\ncMr18PQ1hHoO4dmR/4+/L11DTP0OqjJGc+rE4fzjhN6M6p322dbzFx503RZx4Qtpxl7q5lJ55ho4\n5QZIz3P92OOvhvceg5JCmHBdR9UdmREzjuz5Pt+BW80p2XDLB+4NsTNlDXU/S1a4MfwKdpFDov8x\nAGvnw/a3XRjnjnXLTrgMSlfDf+7DrnqBZuKYVfUdPvhnGRMGDGP2pTM+ewKzPZ//02Dfa8Cp8P0N\n7rG98gvcPCvl66O3v7j9fnaGzAHuqttQIHr/LiJdSBOHBZrdkMScUTD+ms88ZM/6MRV5Z9PU0sKl\nNd8hmNaXp66bxLPXT+bM4Tn7D/YDaRvs4E5qFnwNfLHQf/IR7IjH+GMhc6D7XSNgRA5Z92y5WwvN\nNVBf7i5+qdwKV7/4yUf/qoYWXvpwB48t3cbm0q8yIPlqrr+kgEtPyndXhXa2ibNd90lan87fdjTL\nGupmh+w7oasrEYk63TPcX7rF9XPvNWw6pdmTef3d7SxYtYu3N5YTCFnG5qVzz2XjufCE3iTE+ve/\nvSPl8312mKE4465w0/8mpHd1JSJRp/uNlmmqgXuHuv7vsZexaHuIP2zpzfs73BS7/XokMWNsby4Y\n25sxeWmH1/UiInKUaLTM/qxf4C75P/2HLKztzzeXrGBsXiw/OH84Zw3PYWTvVAW6iES97hfuK5+D\n9H6Upp/A7Y++xeg+aTx/w2TiYnRuWUS8o3slWn05bHodO+aLfP/5lTS0BLl/1jgFu4h4TvdKtTXz\nwAaZF5jM4o/L+PEFIxmSk9rVVYmIdLruFe4rn6c+bQjffyvAOSN7cdWkTpwLRUTkONJ9wr26GLa/\nzZ+rTmJ4rzTunzVOJ05FxLO6TbhXLX8agLcSzuCRayeQHN/9ziWLSPfh+XBvDYaYs3gTu956jI/s\nEO766kXkpnfyJFciIscZTzdfS6oa+dpflmNL1zA7fhuVZ/w3mXu/9EJExMM823IPhSy3PvMBJVWN\n/OnETWD8ZE6Y1dVliYgcE54N97+8vZVlmyu488IRDNyxwH2/Z0p2V5clInJMeDLcN+yu5e5X13HO\nyBwuy9ruvoPzhC93dVkiIsdMdIf72pfgrd9+ZlEwZPneMx+SEh/Dr754AmblMxCXAsOP8BuJRESi\nSHSfUF3+Z9j8hpsud+ylACz+uJTUnUv44+QcskuXwOq/w4gLIS6pi4sVETl2ojvcq4vdz5e/675I\nOaMfLPghf4ubB4W4G8CJOpEqIt1L9Ia7tS7cR34eNi+G579BU1IuZ9W8zDu5l3PKF74NzbWA0Xdw\niki3E73h3lgJgUboNxnGXALPXksC8IvWq7jykrshO6WrKxQR6TLRG+7VRe5nej6MuohQXTk/er2C\nzb1PZ5CCXUS6uegdLbO3vz09D4BlPS/mqeoxzJrYtwuLEhE5Pngg3F2YP7W8iLSEGKaP6d2FRYmI\nHB+iO9z98ZCURSAY4vW1u7nghD4kxPq7ujIRkS4X3eGengc+H+t21VLfEmTSoB5dXZWIyHEhysM9\nH4DCrRUATBigcBcRgWgP9zQX7su3VdInPYE+GYldXJSIyPEhOsM92Ap1uyA9H2sthVsrKFCrXUTk\nE9EZ7rU7wYYgPZ/iykZ21zTIOWZ5AAAJ9klEQVRTMCCzq6sSETluRGe4fzIMMp8V2yoBKOivlruI\nyF5RH+7Lt1aQGh/D8NzUrq1JROQ4ElG4G2OmGWPWG2M2GmNuP8B6lxpjrDGmoPNK7MDecE/Lo3Br\nJeP7Z+L3maP6kiIi0eSg4W6M8QMPANOBUcDlxphRHayXCtwMvNPZRe6juhgSM6kOxvNxaS0F/dXf\nLiLSViQt94nARmvtZmttCzAXmNnBer8A7gGaOrG+joXHuL+3vRJr0clUEZF2Ign3PKCozf3i8LJP\nGGPGA32ttS8faEPGmNnGmEJjTGFZWdkhF/uJ8Bj3wm0V+H2GcX0zDn9bIiIeFEm4d9SZbT950Bgf\ncB9w68E2ZK2dY60tsNYWZGdnR15lezWu5b5+Vy1DslNIiovemYtFRI6GSMK9GGg7j24+sKPN/VRg\nDPCmMWYrMAmYf9ROqjbVQFM1pOdTVttMr/SEo/IyIiLRLJJwXw4MNcYMNMbEAbOA+XsftNZWW2uz\nrLUDrLUDgGXARdbawo43d4RqStzPcLjnpMYflZcREYlmBw13a20AuBFYCKwFnrHWrjbG3GWMueho\nF7iP8DBIm5ZHWV0z2Qp3EZF9RNRZba1dACxot+zO/ax75pGXdQDhcK+Jz6U1WEF2isJdRKS96LtC\n1R8HOaMotW6EjFruIiL7ir5wH38lfGsppfVBQOEuItKR6Av3sLLaZkDhLiLSEYW7iIgHRW+41zUT\nH+MjNV4XMImItBe94V7rhkEao9kgRUTai/pwFxGRfUV3uGuMu4hIh6I33OuayUlTuIuIdCQqw701\nGKKivoXsFE0aJiLSkagM9z11LYCGQYqI7E9UhrvGuIuIHFh0hnud+yY/hbuISMeiMtxLa9RyFxE5\nkKgM973dMlkpcV1ciYjI8Sk6w72umfTEWOJj/F1diojIcSk6w11Xp4qIHFD0hruuThUR2a/oDHd9\nd6qIyAFFZ7irW0ZE5ICiLtzrmwM0tATJUbiLiOxX1IW7rk4VETm46Av3OoW7iMjBRF+4q+UuInJQ\n0RvuGgopIrJfURfufTISmTY6l8wkTT0gIrI/MV1dwKE6d1Qvzh3Vq6vLEBE5rkVdy11ERA5O4S4i\n4kEKdxERD1K4i4h4kMJdRMSDFO4iIh6kcBcR8aCIwt0YM80Ys94Ys9EYc3sHj3/PGLPGGPORMWaR\nMaZ/55cqIiKROmi4G2P8wAPAdGAUcLkxZlS71d4HCqy1JwDPAfd0dqEiIhK5SFruE4GN1trN1toW\nYC4ws+0K1to3rLUN4bvLgPzOLVNERA5FJOGeBxS1uV8cXrY/Xwde6egBY8xsY0yhMaawrKws8ipF\nROSQRBLupoNltsMVjbkKKAB+3dHj1to51toCa21BdnZ25FWKiMghiWTisGKgb5v7+cCO9isZY84B\nfgScYa1t7pzyRETkcETScl8ODDXGDDTGxAGzgPltVzDGjAf+BFxkrS3t/DJFRORQHDTcrbUB4EZg\nIbAWeMZau9oYc5cx5qLwar8GUoBnjTEfGGPm72dzIiJyDEQ0n7u1dgGwoN2yO9v8fk4n1yUiIkdA\nV6iKiHiQwl1ExIMU7iIiHqRwFxHxIIW7iIgHKdxFRDxI4S4i4kEKdxERD1K4i4h4kMJdRMSDFO4i\nIh6kcBcR8SCFu4iIByncRUQ8SOEuIuJBCncREQ9SuIuIeJDCXUTEgxTuIiIepHAXEfEghbuIiAcp\n3EVEPEjhLiLiQQp3EREPUriLiHiQwl1ExIMU7iIiHqRwFxHxIIW7iIgHKdxFRDxI4S4i4kEKdxER\nD1K4i4h4kMJdRMSDFO4iIh4UUbgbY6YZY9YbYzYaY27v4PF4Y8zT4cffMcYM6OxCRUQkcgcNd2OM\nH3gAmA6MAi43xoxqt9rXgUpr7RDgPuDuzi5UREQiF0nLfSKw0Vq72VrbAswFZrZbZybwWPj354Cz\njTGm88oUEZFDERPBOnlAUZv7xcAp+1vHWhswxlQDPYHytisZY2YDs8N364wx6w+naCCr/bY9RvsX\n3bR/0e1437/+kawUSbh31AK3h7EO1to5wJwIXvPABRlTaK0tONLtHK+0f9FN+xfdvLJ/kXTLFAN9\n29zPB3bsbx1jTAyQDlR0RoEiInLoIgn35cBQY8xAY0wcMAuY326d+cBXwr9fCrxurd2n5S4iIsfG\nQbtlwn3oNwILAT/wiLV2tTHmLqDQWjsf+DPwhDFmI67FPutoFk0ndO0c57R/0U37F908sX9GDWwR\nEe/RFaoiIh6kcBcR8aCoC/eDTYUQbYwxfY0xbxhj1hpjVhtjbgkv72GMec0YsyH8M7Oraz1cxhi/\nMeZ9Y8zL4fsDw9NUbAhPWxHX1TUeLmNMhjHmOWPMuvAx/JzHjt13w/8uVxljnjLGJETz8TPGPGKM\nKTXGrGqzrMPjZZzfh7PmI2PMSV1X+aGLqnCPcCqEaBMAbrXWjgQmAd8O79PtwCJr7VBgUfh+tLoF\nWNvm/t3AfeF9q8RNXxGt7gdetdaOAE7E7acnjp0xJg+4GSiw1o7BDaiYRXQfv0eBae2W7e94TQeG\nhm+zgQePUY2dIqrCncimQogq1tqd1tr3wr/X4sIhj89O6fAY8IWuqfDIGGPygQuAh8P3DTAVN00F\nRPe+pQGn40aLYa1tsdZW4ZFjFxYDJIavX0kCdhLFx89au5h9r8HZ3/GaCTxunWVAhjGm97Gp9MhF\nW7h3NBVCXhfV0unCs2mOB94Bellrd4J7AwByuq6yI/I74IdAKHy/J1BlrQ2E70fzMRwElAF/CXc7\nPWyMScYjx85aWwLcC2zHhXo1sALvHL+99ne8ojpvoi3cI5rmIBoZY1KA54HvWGtrurqezmCMuRAo\ntdauaLu4g1Wj9RjGACcBD1prxwP1RGkXTEfCfc8zgYFAHyAZ11XRXrQev4OJ6n+r0RbukUyFEHWM\nMbG4YP+rtfaF8OLdez8Chn+WdlV9R2AKcJExZiuuC20qriWfEf6YD9F9DIuBYmvtO+H7z+HC3gvH\nDuAcYIu1tsxa2wq8AEzGO8dvr/0dr6jOm2gL90imQogq4T7oPwNrrbW/bfNQ2ykdvgL8/VjXdqSs\ntXdYa/OttQNwx+p1a+2VwBu4aSogSvcNwFq7CygyxgwPLzobWIMHjl3YdmCSMSYp/O907/554vi1\nsb/jNR+4JjxqZhJQvbf7JipYa6PqBswAPgY2AT/q6no6YX9OxX3U+wj4IHybgeubXgRsCP/s0dW1\nHuF+ngm8HP59EPAusBF4Fojv6vqOYL/GAYXh4zcPyPTSsQN+DqwDVgFPAPHRfPyAp3DnD1pxLfOv\n7+944bplHghnzUrcqKEu34dIb5p+QETEg6KtW0ZERCKgcBcR8SCFu4iIByncRUQ8SOEuIuJBCncR\nEQ9SuIuIeND/B3wjnICZgj9uAAAAAElFTkSuQmCC\n",
      "text/plain": [
       "<matplotlib.figure.Figure at 0x7f623242a748>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "epochs = 110\n",
    "\n",
    "a, b = [], []\n",
    "for epoch in range(epochs):\n",
    "    if epoch == 50:\n",
    "        trainer.set_learning_rate(0.01)\n",
    "    if epoch == 80:\n",
    "        trainer.set_learning_rate(0.001)\n",
    "    train_loss = 0.\n",
    "    train_acc = 0.\n",
    "    start = time()\n",
    "    for data, label in train_data:\n",
    "        data = augment(data, aug_train).as_in_context(ctx)\n",
    "        label = label.as_in_context(ctx)\n",
    "        with autograd.record():\n",
    "            output = net(data)\n",
    "            l = loss(output, label)\n",
    "        l.backward()\n",
    "        trainer.step(batch_size)\n",
    "        train_loss = train_loss + nd.mean(l).asscalar()\n",
    "        train_acc = train_acc + accuracy(output, label)\n",
    "    test_acc = evaluate_accuracy(test_data, net, ctx)\n",
    "    \n",
    "    if epoch%10 == 0:\n",
    "        print(epoch, 'loss:%.4f tracc:%.4f teacc:%.4f time:%.3f'%(\n",
    "            train_loss/len(train_data), train_acc/len(train_data), test_acc, time()-start)) \n",
    "    a.append(train_acc/len(train_data))\n",
    "    b.append(test_acc)\n",
    "\n",
    "print('tracc:%f teacc:%f'%(train_acc/len(train_data), test_acc))\n",
    "plt.plot(np.arange(epochs), a, np.arange(epochs), b)\n",
    "plt.ylim(0,1)\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.4"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}
